package cn.softgame.crawler;

import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.HashSet;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
import javax.xml.xpath.XPath;
import javax.xml.xpath.XPathConstants;
import javax.xml.xpath.XPathExpression;
import javax.xml.xpath.XPathExpressionException;
import javax.xml.xpath.XPathFactory;

import org.htmlparser.Node;
import org.htmlparser.NodeFilter;
import org.htmlparser.Parser;
import org.htmlparser.filters.NodeClassFilter;
import org.htmlparser.filters.OrFilter;
import org.htmlparser.tags.LinkTag;
import org.htmlparser.util.NodeList;
import org.htmlparser.util.ParserException;
import org.w3c.dom.Document;
import org.xml.sax.SAXException;

public class HtmlParserTool {
	
	// 获取一个网站上的链接,filter 用来过滤链接
	public static Set<String> extracLinks(String url, LinkFilter filter) {
		Set<String> links = new HashSet<String>();
		if(url==null || "".equals(url))
			return links;
		try {
			Parser parser = new Parser(url);
			String encoding = parser.getEncoding();
			System.out.println("encoding="+encoding);
			parser.setEncoding("GB2312");
			// 过滤 <frame >标签的 filter，用来提取 frame 标签里的 src 属性所表示的链接
			NodeFilter frameFilter = new NodeFilter() {
				private static final long serialVersionUID = -7260954926577570185L;
				public boolean accept(Node node) {
					if (node.getText().toLowerCase().startsWith("frame src=")) {
						return true;
					} else {
						return false;
					}
				}
			};
			// OrFilter 来设置过滤 <a> 标签，和 <frame> 标签
			OrFilter linkFilter = new OrFilter(new NodeClassFilter(LinkTag.class), frameFilter);
			// 得到所有经过过滤的标签
			NodeList list = parser.extractAllNodesThatMatch(linkFilter);
			for (int i = 0; i < list.size(); i++) {
				Node tag = list.elementAt(i);
				if (tag instanceof LinkTag) {// <a> 标签
					LinkTag link = (LinkTag) tag;
					String linkUrl = link.getLink();// url
					if (filter.accept(linkUrl))
						links.add(linkUrl);
				} else {// <frame> 标签
					// 提取 frame 里 src 属性的链接如 <frame src="test.html"/>
					String frame = tag.getText();
					int start = frame.indexOf("src=");
					frame = frame.substring(start);
					int end = frame.indexOf(" ");
					if (end == -1)
						end = frame.indexOf(">");
					String frameUrl = frame.substring(5, end - 1);
					if (filter.accept(frameUrl))
						links.add(frameUrl);
				}
			}
		} catch (ParserException e) {
			e.printStackTrace();
		}
		return links;
	}

	// 测试的 main 方法
	public static void main(String[] args) {
		/*Set<String> links = HtmlParserTool.extracLinks("D:\\crawler\\www.programmer.com.cn.html",
				new LinkFilter() {
					public boolean accept(String url) {
						return true;
					}

				});
		for (String link : links)
			System.out.println(link);*/
		
		File file = new File("D:\\crawler\\www.csdn.net.html");
		parseHtml(file);
	}
	
	public static Set<String> parseHtml(File file){
		Set<String> links = new HashSet<String>();
		try {
			InputStream is = new FileInputStream(file);
			InputStreamReader isr = new InputStreamReader(is);
			StringBuffer sb = new StringBuffer();
			int ch=0;
			while((ch=isr.read())!=-1){
				sb.append((char)ch);
			}
			String regex = "http://[\\w\\.\\-/:]+";
			// String regex = "<a\\s+href\\s*=\\s*(\"[^\"]*\"|[^\\s>]|\"[^\"]*\"\\s*+[^>\"]*\\s*=\\s*(\"[^\"]*\"|[^\\s>]))\\s*>";
			 Pattern pattern = Pattern.compile(regex,Pattern.CASE_INSENSITIVE);
			 Matcher matcher = pattern.matcher(sb);
			 int i=0;
			 while(matcher.find()){
				 int start = matcher.start();
				 int end = matcher.end();
				 String matchUrl = sb.substring(start, end);
				 System.out.println(++i + ":"+matchUrl);
				 links.add(matchUrl);
			 }
		} catch (FileNotFoundException e) {
			e.printStackTrace();
		} catch (IOException e) {
			e.printStackTrace();
		}
		return links;
	}
	
}
